# coding: utf8

import os,sys
sys.path.append('..')

import logging
import re
import random
import time
import requests_html
import traceback
from collections import defaultdict
from multiprocessing.pool import ThreadPool
from server.post_data import PostHelper


""" 北京航空航天大学
"""

logging.basicConfig(level=logging.INFO)

session = requests_html.HTMLSession()
pool = ThreadPool(os.cpu_count() * 2)

reduce_nonalpha = re.compile(r"\s{2,}")

gender_ptn = re.compile(r"性别：([\s\S]*?\n)")
birth_ptn = re.compile(r"出生年月：([\s\S]*?\n)")
grad_ptn = re.compile(r"毕业院校：([\s\S]*?\n)")
degree_ptn = re.compile(r"学位：([\s\S]*?\n)")
school_ptn = re.compile(r"所在单位：([\s\S]*?\n)")
subject_ptn = re.compile(r"学科：([\s\S]*?\n)")
addr_ptn = re.compile(r"办公地点：([\s\S]*?\n)")
tel_ptn = re.compile(r"联系方式：([\s\S]*?\n)")
email_ptn = re.compile(r"Email：([\S]*?\n)")

post_helper = PostHelper()


def teacher_shi_list():
    # http://www.sme.buaa.edu.cn/szdw/szjj.htm
    try:
        r = session.get("http://www.sme.buaa.edu.cn/szdw/szjj.htm")
        if r.status_code == 200:
            container = r.html.xpath("//div[contains(@class,'Pic')][1]")[0]
            uls = container.xpath("//ul")

            teachers = []

            for ul in uls:
                position = ul.xpath("//h2[1]/text()")
                position = position[0] if position else ''
                print(position)

                lis = ul.xpath("//li")
                for li in lis:
                    url = list(li.xpath("//a")[0].absolute_links)[0]
                    avatar = li.xpath("//a/img/@src")
                    avatar = "http://www.sme.buaa.edu.cn" + avatar[0] if avatar else ''
                    name = li.text

                    logging.info("pos: %s, name: %s, avatar: %s, url: %s", position, name, avatar, url)
                    teachers.append((name, url, position, avatar))

            pool.imap_unordered(teacher_detail, teachers)
            pass
        else:
            logging.warning("status code error: %s", r.status_code)
            r.raise_for_status()

    except Exception as e:
        traceback.print_exc()
        raise e


def teacher_list():
    headers = {"Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Encoding": "gzip, deflate", "Accept-Language": "zh-CN,zh;q=0.9", "Cookie": "JSESSIONID=8E408D7A10980F24A01CF2A4A4D0695C", "Host": "shi.buaa.edu.cn", "Referer": "http://shi.buaa.edu.cn/jsjs.jsp?urltype=tree.TreeTempUrl&wbtreeid=1012", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"}
    payloads = {"collegeid": 1176, "disciplineid": 0, "enrollid": 0, "pageindex": 1, "pagesize": 12, "rankid": 0, "honorid": 0, "pinyin": '', "profilelen": 100, "teacherName": '', "searchDirection": '', "viewmode": 8, "viewid": 65073, "siteOwner": 1211900872, "viewUniqueId": "u10", "showlang": "zh_CN", "ellipsis": "...", "alignright": False}

    tot_pages = 0  # 数值型的，不能传入 Inner function.
    cur_page = 1

    def get_list(tot_pages, cur_page):
        time.sleep(random.randint(0, 5))
        try:
            payloads['pageindex'] = cur_page
            r = session.get("http://shi.buaa.edu.cn/system/resource/tsites/advancesearch.jsp", headers=headers, params=payloads)
            if r.status_code == 200:
                res = r.json()

                if tot_pages < 1:
                    tot_pages = res.get("totalpage", 0)
                    cur_page = res.get("pageindex", 1)

                teachers = []

                for teacher in res['teacherData']:
                    name = teacher.get("ename", "")
                    name_zh = teacher['teacherName']
                    position = " ".join([teacher.get("prorank", ""), teacher.get("gtutor", ""), teacher.get("doctorTutor", "")])
                    gender = teacher.get("sex", "")
                    avatar = "http://shi.buaa.edu.cn" + teacher['picUrl'] if 'picUrl' in teacher else ''
                    url = teacher['url']

                    logging.info("page: %d/%d, name_zh: %s. name: %s, gender: %s, position: %s, url: %s", cur_page, tot_pages, name_zh, name, gender, position, url)

                    teachers.append((name_zh, url, position, avatar))

                pool.imap_unordered(teacher_detail, teachers)
            else:
                logging.warning("status code error: %s", r.status_code)
                r.raise_for_status()

        except Exception as e:
            traceback.print_exc()
            raise e

        return tot_pages, cur_page

    tot_pages, cur_page = get_list(tot_pages, cur_page)
    for page in range(cur_page + 1, tot_pages + 1):
        get_list(tot_pages, page)
    pass


def decode_email(id, content):
    headers = {"Accept": "application/json, text/javascript, */*; q=0.01"}
    payloads = {"id": id, "content": content, "mode": 8}  # "id="+id+"&content="+content+"&mode="+_tsites_com_view_mode_type_,
    try:
        r = session.get("http://shi.buaa.edu.cn/system/resource/tsites/tsitesencrypt.jsp", headers=headers, params=payloads)
        if r.status_code == 200:
            data = r.json()
            return data['content']
        else:
            logging.warning("status code error: %s", r.text)
            r.raise_for_status()
    except Exception as e:
        traceback.print_exc()
        raise e
        pass


def teacher_detail(payload):
    name, url, position, avatar = payload
    time.sleep(random.randint(0, 5))

    logging.info("loading %s ...", name)

    def detail_theme1(r):
        infos = r.html.xpath('//div[@id="leftside"]//div[contains(@class, "jbqk")][1]')[0].full_text
        infos = reduce_nonalpha.sub("\n", infos)
        gender, birth, grad, degree, school, subject, addr, tel, email = [ptn.search(infos) for ptn in (gender_ptn, birth_ptn, grad_ptn, degree_ptn, school_ptn, subject_ptn, addr_ptn, tel_ptn, email_ptn)]
        gender, birth, grad, degree, school, subject, addr, tel, email = [(ptn.group(1).strip("\n") if ptn else '') for ptn in (gender, birth, grad, degree, school, subject, addr, tel, email)]

        email_span_id = r.html.xpath('//div[@id="leftside"]//div[contains(@class, "jbqk")][1]//span[@_tsites_encrypt_field="_tsites_encrypt_field"][1]/@id')[0]
        email = decode_email(email_span_id, email)  # 31540465377523d5d71d64aa896aae3be6a883efc2e9b9356087abaf64bf9c6ea456a0aa4af2599297d360f82bc365b3075c1d13c21e48e86bc414b76d9635abc0b54a35aa5297923aabb3517e7897d8749c4b2b9e0ac55375d91c1d8c577d708d149f4b89d90340953e5da92d926a9d847253e2772ada34cf05f0e0e56a749c

        # print(name, gender, birth, grad, degree, school, subject, addr, tel, email)

        otherinfos = r.html.xpath('//div[@id="rightside"]//*[@class="content"]//div[@class="con_bload"]')
        others = defaultdict(str)
        for info in otherinfos:
            header = info.xpath("//h2/text()")[0]
            content = reduce_nonalpha.sub(' ', info.full_text).replace(header, '', 1).strip('\n')

            # print("%s: %s" % (header, content))
            others[header] = content

        # print(name, gender, birth, grad, degree, school, subject, addr, tel, email, others.get("个人简历", ''), others.get('教育经历', ''), others.get('工作经历', ''), others.get('研究方向', ''))

        post_helper.post_expert_data(name, '北京航空航天大学', school, position, degree, tel, email, addr, avatar, others.get('教育经历', ''), others.get('工作经历', ''), others.get('研究方向', ''), '', others.get('个人简历', ''), '北京航空航天大学', url)

        pass

    def detail_theme2(r):
        ptn = r'个人简[介|历]([\s\S]*)?(版权所有|手机版)'

        # print(r.encoding)  # default charset is ISO-8859-1
        r.encoding = r.apparent_encoding  # apparent_encoding is UTF8
        # print(r.encoding)

        text = reduce_nonalpha.sub(' ', r.html.full_text)

        content = re.search(ptn, text)
        content = content.group(1) if content else ""

        # print(name, ", ", content)
        pass

    try:
        r = session.get(url, headers={'Content-Type': 'text/html'})
        if r.status_code == 200:
            if r.html.xpath("//div[@id='leftside']"):
                detail_theme1(r)
            else:
                detail_theme2(r)
            pass
        else:
            logging.warning("status code error: %s", r.status_code)
            r.raise_for_status()

    except Exception as e:
        traceback.print_exc()
        raise e


if __name__ == '__main__':
    teacher_list()

    pool.close()
    pool.join()

    session.close()

    post_helper.close()
    pass


"""
[1]瑞士洛桑联邦理工大学（EPFL）
[2]慕尼黑工业大学
[3]新西兰惠灵顿维多利亚大学
[4]巴黎第十一大学
[5]美国国家标准与技术研究院
[6]美国科罗拉多州立大学UCLA
[7]德国马克斯-普朗克研究所
[8]北京大学
[9]中科院半导体研究所
[10]电子科技大学
[11]台湾中央研究院
[12]中科院微电子所
[13]南方科技大学
[14]复旦大学
[15]电子科技大学
"""

